package com.crawler;

import com.crawler.model.PivixPojo;
import net.lightbody.bmp.proxy.jetty.util.IO;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.openqa.selenium.Keys;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.interactions.Actions;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;

public class Main {

    private static String proxyServer = "http://127.0.0.1:10809"; //本地梯子代理

    private static String mainWeb = "https://www.pixiv.net/ranking.php?mode=daily&content=illust"; //爬取开始的页面，如果需要登录可以先登录一次后重新运行。

    private static String localPath = "F:\\pivix";

    private static HashMap<String,String> headers = new HashMap<>();
    private static Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 10809));

    static {
        headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36");
        headers.put("Accept","*/*");
        headers.put("Accept-Language","zh-CN,zh;q=0.9");
        headers.put("Accept-Encoding","");
        headers.put("Connection","keep-alive");
    }

    public static void main(String[] args) throws Exception {

        ChromeOptions options = new ChromeOptions();
        //设置从ChromeDriver中获取属性（处理反爬机制）
        //设置谷歌浏览器用户数据目录
        options.addArguments("--user-data-dir=" + "D:\\Users\\Administrator\\AppData\\Local\\Google\\Chrome\\User Data");
        options.addArguments("--disable-gpu");
        options.addArguments("start-maximized");
        options.addArguments("enable-automation");
        options.addArguments("--no-sandbox");
        options.addArguments("--disable-infobars");
        options.addArguments("--disable-dev-shm-usage");
        options.addArguments("--disable-browser-side-navigation");

        options.addArguments("--proxy-server=" + proxyServer);
        ChromeDriver driver = new ChromeDriver(options);
        driver.get(mainWeb);

        //休眠一下等浏览器启动
        Thread.sleep(1000);
        //使用Jsoup来解析页面
        Document document = Jsoup.parse(driver.getPageSource());
        //根据class来获取也没上的元素
        Elements elements = document.body().getElementsByClass("ranking-item");

        List<PivixPojo> pivixPojoList = elements.stream()
                .map(element -> {
                    return PivixPojo.builder()
                            .dataTitle(element.attr("data-title"))
                            .dataUserName(element.attr("data-user-name"))
                            .dataDate(element.attr("data-date"))
                            .dataViewCount(Integer.parseInt(element.attr("data-view-count")))
                            .dataRationgCount(Integer.parseInt(element.attr("data-rating-count")))
                            .dataId(element.attr("data-id"))
                            .build();
                }).collect(Collectors.toList());

        pivixPojoList.forEach(pivixPojo -> {
            doDownLoad(driver, pivixPojo, localPath);
        });
    }


    public static void doDownLoad(ChromeDriver driver, PivixPojo pivixPojo, String path){
        try{
            driver.get("https://www.pixiv.net/artworks/" + pivixPojo.getDataId());

            Thread.sleep(1000);

            Document document = Jsoup.parse(driver.getPageSource());

            Elements elements = document.getElementsByTag("main").get(0).getElementsByTag("section").get(0).getElementsByTag("figure").get(0).getElementsByTag("a");

            elements.stream().forEach(element -> {

                String url = element.attr("href"); //高清图片
                String masterUrl = element.getElementsByTag("img").get(0).attr("src"); //压缩的图片
                downLoadImage(Paths.get(path, UUID.randomUUID().toString() + url.substring(url.lastIndexOf("."))).toString(), url);
                downLoadImage(Paths.get(path, UUID.randomUUID().toString() + url.substring(url.lastIndexOf("."))).toString(),  masterUrl);

            });

        }catch (Exception e){
            e.printStackTrace();
        }


    }

    private static void downLoadImage(String path, String url) {

        Connection connection = Jsoup
                .connect(url)
                .headers(headers)
                .referrer("https://www.pixiv.net/")
                .proxy(proxy)
                .ignoreContentType(true)
                .maxBodySize(104857600) //设置运行下载的图片大小
                // 超时时间一分钟，防止网络延迟的影响
                .timeout(6000000);
        try {
            IO.copy(connection.execute().bodyStream(), new FileOutputStream(new File(path)));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }


    private static void doDropDown(ChromeDriver driver) { //下拉加载更多
        String height = driver.executeScript("return document.body.scrollHeight;").toString();
        // 按下键盘的END键滑动键盘,下拉页面
        Actions actions = new Actions(driver);
        actions.sendKeys(Keys.END).perform();
        try {
            Thread.sleep(4000);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
